"""
This crawler crawling the 2dfan,a website introducing galageme information
"""
from urllib.request import urlopen
from urllib.request import Request
from urllib.error import URLError, HTTPError
from bs4 import BeautifulSoup
import os
import re

indexUrl = "http://www.2dfan.com/subjects/"
index = 1
errorTimes = 0
tolerateErrorTimes = 50

user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"
headers = {"User-Agent": user_agent}

cmd_mkdir = "mkdir output"
os.system(cmd_mkdir)

while True:
    # condition judgement of loop ending
    if errorTimes >= tolerateErrorTimes:
        break

    # send the request and open the web page
    try:
        url = indexUrl + str(index)
        webpage = Request(url, None, headers)
        html = urlopen(webpage)
    except URLError as e1:
        print("(%d)url open ERROR!,reason is %s" %(index, e1.reason))
        index += 1
        continue
    except HTTPError as e2:
        print("(%d)url open ERROR!,error code is %d,reason is %s,header is%s" %(index, e2.code, e2.reason, e2.headers))
        index += 1
        continue
    bsObj = BeautifulSoup(html, "html.parser")

    # crawling
    try:
        title = bsObj.find("div", {"class": "block"}).h3.get_text()
        product = bsObj.find_all("p", {"class":"tags"})[0].a.get_text()
        score = bsObj.find('em', {'class': 'score'}).get_text()
        votepnum = bsObj.find('div', {'class':'rank-info'}).find('span', {'class':'muted'}).get_text()
        print("(%d)%s %s %s %s" %(index, title, product, score, votepnum))
        index += 1
        errorTimes = 0
    except AttributeError as e:
        print("(%d)ERROR!,reason is %s" % (index, e.reason))
        index += 1
        errorTimes += 1
        continue
logf.close()
print("COMPLETE!TOTAL: %d OBJECTS" %(index - errorTimes))